#!/usr/bin/env python3

import requests
import json
from urllib.parse import urlencode
from config import *
import pymongo
from multiprocessing import Pool
import os
import time

#配置mongodb的信息
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_Field_NAME_DB]


def conn_microc_academic(name_id,sec_name_id,ori_name_id):
    #url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=' + str(name_id)
    #url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=71924100'

    #url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath='+ str(ori_name_id) + '%2c' + str(name_id)
    #url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=71924100%2C126322002'


    #db['field_name_four'].save({'name':field_data_item.get('name'),'name_id':field_data_item.get('id'),'thr_name':data_item.get('name'),'thr_name_id':data_item.get('name_id'),'sec_name':data_item.get('sec_name'),'sec_name_id':data_item.get('sec_name_id'),'ori_name':data_item.get('ori_name'),'ori_name_id':data_item.get('ori_name_id')})
    #conn_microc_academic(data_item.get('name_id'),data_item.get('sec_name_id'),data_item.get('ori_name_id'))


    url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath='+ str(ori_name_id) +'%2C'+ str(sec_name_id) +'%2c' + str(name_id)
    #url = 'https://academic.microsoft.com/api/analytics/topics/hierarchy?topicPath=71924100%2C126322002%2C170493617'
    #print("11111111111111111111111111111111111111111")
    try:
        response = requests.get(url)
        #print(response)
        if response.status_code == 200:
        #    print(111111111111111111)
        #    print(response.content)
            data_json = json.loads(response.content)
        #    print("--------------------------")
        #    print(data_json)
            return data_json['childTopics'],response.status_code
        else:
            print("到底了!")
            return [{}],response.status_code

    except requests.ConnectionError as e:
        print('出错',e.args)
        return [{}],403


def get_field_name_one():
    field_name_one_data = db['field_name_three'].find({}).batch_size(1)
    return field_name_one_data

def do_conn(field_name_one_data):
    field_name_fail_list = []
    for data_item in field_name_one_data:
        #print(data_item.get('name_id'))
        micro_academic_field_data,status_code = conn_microc_academic(data_item.get('name_id'),data_item.get('sec_name_id'),data_item.get('ori_name_id'))
        if status_code != 403:
            for field_data_item in micro_academic_field_data:

                db['field_name_four'].save({'name':field_data_item.get('name'),'name_id':field_data_item.get('id'),'thr_name':data_item.get('name'),'thr_name_id':data_item.get('name_id'),'sec_name':data_item.get('sec_name'),'sec_name_id':data_item.get('sec_name_id'),'ori_name':data_item.get('ori_name'),'ori_name_id':data_item.get('ori_name_id')})
    #pool = Pool()
        else:
            field_name_fail_list.append(data_item)

    return field_name_fail_list

def do_conn_pool(data_item):
    print(data_item)
    micro_academic_field_data,status_code = conn_microc_academic(data_item.get('name_id'),data_item.get('sec_name_id'),data_item.get('ori_name_id'))
    if status_code != 403:
        for field_data_item in micro_academic_field_data:

            db['field_name_four'].save({'name':field_data_item.get('name'),'name_id':field_data_item.get('id'),'thr_name':data_item.get('name'),'thr_name_id':data_item.get('name_id'),'sec_name':data_item.get('sec_name'),'sec_name_id':data_item.get('sec_name_id'),'ori_name':data_item.get('ori_name'),'ori_name_id':data_item.get('ori_name_id')})
    #pool = Pool()
        return None
    else:
        return data_item


def main():
    field_name_one_data = get_field_name_one()
    #field_name_fail_list = do_conn(field_name_one_data)
    field_list = []
    field_name_fail_list = []
    pool = Pool()
    field_list = pool.map(do_conn_pool,field_name_one_data)
    for data_item in field_list:
        if data_item:
            field_name_fail_list.append(data_item)
    print(field_name_fail_list)
    while len(field_name_fail_list) != 0:
        print(len(field_name_fail_list))
        field_name_fail_list = do_conn(field_name_fail_list)



if __name__ == '__main__':

    main()
